{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import argparse\n",
    "import json\n",
    "from collections import defaultdict, Counter\n",
    "import re\n",
    "\n",
    "MAX_WORDS = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Copyright (c) Facebook, Inc. and its affiliates.\n",
    "\"\"\"\n",
    "\n",
    "\"\"\"This file has functions to preprocess the chat from user before\n",
    "querying the dialogue manager\"\"\"\n",
    "import string\n",
    "\n",
    "from spacy.lang.en import English\n",
    "from typing import List\n",
    "\n",
    "tokenizer = English().Defaults.create_tokenizer()\n",
    "\n",
    "\n",
    "def word_tokenize(st) -> str:\n",
    "    chat_with_spaces = insert_spaces(st)\n",
    "    return \" \".join([str(x) for x in tokenizer(chat_with_spaces)])\n",
    "\n",
    "\n",
    "def sentence_split(st):\n",
    "    st = st.replace(\" ?\", \" .\")\n",
    "    st = st.replace(\" !\", \" .\")\n",
    "    st = st.replace(\" ...\", \" .\")\n",
    "    res = [\n",
    "        \" \".join([x for x in sen.lower().split() if x not in string.punctuation])\n",
    "        for sen in st.split(\" .\")\n",
    "    ]\n",
    "    return [x for x in res if x != \"\"]\n",
    "\n",
    "\n",
    "def insert_spaces(chat):\n",
    "    updated_chat = \"\"\n",
    "    for i, c in enumerate(chat):\n",
    "        # [num , (num , {num , ,num , :num\n",
    "        if (\n",
    "            (c in [\"[\", \"(\", \"{\", \",\", \":\", \"x\"])\n",
    "            and (i != len(chat) - 1)\n",
    "            and (chat[i + 1].isdigit())\n",
    "        ):\n",
    "            updated_chat += c + \" \"\n",
    "        # num, , num] , num) , num}, num:\n",
    "        # 4x -> 4 x\n",
    "        elif (\n",
    "            (c.isdigit())\n",
    "            and (i != len(chat) - 1)\n",
    "            and (chat[i + 1] in [\",\", \"]\", \")\", \"}\", \":\", \"x\"])\n",
    "        ):\n",
    "            updated_chat += c + \" \"\n",
    "        else:\n",
    "            updated_chat += c\n",
    "\n",
    "    return updated_chat\n",
    "\n",
    "\n",
    "def preprocess_chat(chat: str) -> List[str]:\n",
    "    # For debug mode, return as is.\n",
    "    if chat == \"_debug_\" or chat.startswith(\"_ttad_\"):\n",
    "        return [chat]\n",
    "\n",
    "    # Tokenize\n",
    "    tokenized_line = word_tokenize(chat)\n",
    "    tokenized_sentences = [sen for sen in sentence_split(tokenized_line)]\n",
    "\n",
    "    return tokenized_sentences\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_repeat_dict(d):\n",
    "    if d[\"loop\"] == \"ntimes\":\n",
    "        repeat_dict = {\"repeat_key\": \"FOR\"}\n",
    "        processed_d = process_dict(with_prefix(d, \"loop.ntimes.\"))\n",
    "        if 'repeat_for' in processed_d:\n",
    "            repeat_dict[\"repeat_count\"] = processed_d[\"repeat_for\"]\n",
    "        if 'repeat_dir' in processed_d:\n",
    "            repeat_dict['repeat_dir'] = processed_d['repeat_dir']\n",
    "        return repeat_dict\n",
    "    if d[\"loop\"] == \"repeat_all\":\n",
    "        repeat_dict = {\"repeat_key\": \"ALL\"}\n",
    "        processed_d = process_dict(with_prefix(d, \"loop.repeat_all.\"))\n",
    "        if 'repeat_dir' in processed_d:\n",
    "            repeat_dict['repeat_dir'] = processed_d['repeat_dir']\n",
    "        return repeat_dict\n",
    "    if d[\"loop\"] == \"forever\":\n",
    "        return {\"stop_condition\": {\"condition_type\": \"NEVER\"}}\n",
    "    if d['loop'] == 'repeat_until':\n",
    "        stripped_d = with_prefix(d, 'loop.repeat_until.')\n",
    "        if not stripped_d:\n",
    "            return None\n",
    "        processed_d = process_dict(stripped_d)\n",
    "        if 'adjacent_to_block_type' in processed_d:\n",
    "            return {\"stop_condition\" : {\n",
    "                        \"condition_type\" : 'ADJACENT_TO_BLOCK_TYPE',\n",
    "                        'block_type': processed_d['adjacent_to_block_type']}\n",
    "                   }\n",
    "        elif 'condition_span' in processed_d:\n",
    "            return {\"stop_condition\" : {\n",
    "                        \"condition_span\" : processed_d['condition_span']}\n",
    "                   }\n",
    "    \n",
    "    raise NotImplementedError(\"Bad repeat dict option: {}\".format(d[\"loop\"]))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_get_memory_dict(d):\n",
    "    filters_val = d['filters']\n",
    "    out_dict = {'filters': {}}\n",
    "    parent_dict = {}\n",
    "    if filters_val.startswith('type.'):\n",
    "        parts = remove_prefix(filters_val, 'type.').split('.')\n",
    "        type_val = parts[0]\n",
    "        if type_val in ['ACTION', 'AGENT']:\n",
    "            out_dict['filters']['temporal'] = 'CURRENT'\n",
    "            tag_val = parts[1]\n",
    "            out_dict['answer_type'] = 'TAG'\n",
    "            out_dict['tag_name'] = parts[1] # the name of tag is here\n",
    "            if type_val == 'ACTION':\n",
    "                x = with_prefix(d, 'filters.'+filters_val+'.')\n",
    "                out_dict['filters'].update(x)\n",
    "        elif type_val in ['REFERENCE_OBJECT']:\n",
    "            d.pop('filters')\n",
    "            ref_obj_dict = remove_key_prefixes(d, ['filters.type.'])\n",
    "            ref_dict = process_dict(ref_obj_dict)\n",
    "            if 'answer_type' in ref_dict['reference_object']:\n",
    "                out_dict['answer_type'] = ref_dict['reference_object']['answer_type']\n",
    "                ref_dict['reference_object'].pop('answer_type')\n",
    "            if 'tag_name' in ref_dict['reference_object']:\n",
    "                out_dict['tag_name'] = ref_dict['reference_object']['tag_name']\n",
    "                ref_dict['reference_object'].pop('tag_name')    \n",
    "            out_dict['filters'].update(ref_dict)\n",
    "            \n",
    "        out_dict['filters']['type'] = type_val\n",
    "        \n",
    "    return out_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_prefix(text, prefix):\n",
    "    if text.startswith(prefix):\n",
    "        return text[len(prefix):]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_get_memory(d):\n",
    "    out_d = {'dialogue_type': 'GET_MEMORY'}\n",
    "    child_d = process_get_memory_dict(with_prefix(d, \"action_type.ANSWER.\"))\n",
    "    out_d.update(child_d)\n",
    "    return out_d\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert s to snake case\n",
    "def snake_case(s):\n",
    "    return re.sub(\"([a-z])([A-Z])\", \"\\\\1_\\\\2\", s).lower()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''this function splits the key that starts with a given prefix and only for values that are not None\n",
    "and makes the key be the thing after prefix\n",
    "'''\n",
    "def with_prefix(d, prefix):\n",
    "    return {\n",
    "        k.split(prefix)[1]: v\n",
    "        for k, v in d.items()\n",
    "        if k.startswith(prefix) and v not in (\"\", None, \"None\")\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' this function removes certain prefixes from keys and renames the key to be: key with text following \n",
    "the prefix in the dict'''\n",
    "def remove_key_prefixes(d, ps):\n",
    "    \n",
    "    for p in ps:\n",
    "        d = d.copy()\n",
    "        rm_keys = []\n",
    "        add_items = []\n",
    "        # print(p, d)\n",
    "        for k, v in d.items():\n",
    "            if k.startswith(p):\n",
    "                rm_keys.append(k)\n",
    "                add_items.append((k[len(p) :], v))\n",
    "        for k in rm_keys:\n",
    "            del d[k]\n",
    "        for k, v in add_items:\n",
    "            d[k] = v\n",
    "    return d\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_spans_due_to_empty_words(action_dict, words):\n",
    "    \"\"\"Return modified (action_dict, words)\"\"\"\n",
    "\n",
    "    def reduce_span_vals_gte(d, i):\n",
    "        for k, v in d.items():\n",
    "            if type(v) == dict:\n",
    "                reduce_span_vals_gte(v, i)\n",
    "                continue\n",
    "            try:\n",
    "                a, b = v\n",
    "                if a >= i:\n",
    "                    a -= 1\n",
    "                if b >= i:\n",
    "                    b -= 1\n",
    "                d[k] = [[a, b]]\n",
    "            except ValueError:\n",
    "                pass\n",
    "            except TypeError:\n",
    "                pass\n",
    "\n",
    "    # remove trailing empty strings\n",
    "    while words[-1] == \"\":\n",
    "        del words[-1]\n",
    "\n",
    "    # fix span\n",
    "    i = 0\n",
    "    while i < len(words):\n",
    "        if words[i] == \"\":\n",
    "            reduce_span_vals_gte(action_dict, i)\n",
    "            del words[i]\n",
    "        else:\n",
    "            i += 1\n",
    "\n",
    "    return action_dict, words\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "from operator import itemgetter\n",
    "\n",
    "def process_dict(d):\n",
    "    r = {}\n",
    "\n",
    "    # remove key prefixes\n",
    "    # maybe more keys here\n",
    "#     print(d)\n",
    "#     print(\"----------------\")\n",
    "    d = remove_key_prefixes(d, [\"TURN_CHECK.LOOK.\",\n",
    "                                \"TURN_CHECK.POINT.\",\n",
    "                                \"TURN_CHECK.TURN.\",\n",
    "                                \"MOVE.yes.\", \n",
    "                                \"MOVE.no.\", \n",
    "                                \"COPY.yes.\", \n",
    "                                \"COPY.no.\", \n",
    "                                \"receiver_loc.\",\n",
    "                                \"receiver_ref.\",\n",
    "                                \"source_loc.\",\n",
    "                                \"source_ref.\",\n",
    "                                'FREEBUILD.BUILD.', \n",
    "                                'answer_type.TAG.', \n",
    "                                'FREEBUILD.FREEBUILD.', \n",
    "                                'coref_resolve_check.yes.', \n",
    "                                'coref_resolve_check.no.'])\n",
    "#     print(d)\n",
    "#     print(\"----------------new------------------\")\n",
    "    if \"location\" in d:\n",
    "        # print('yes')\n",
    "        r[\"location\"] = {\"location_type\": d[\"location\"]}\n",
    "        if r['location']['location_type'] == 'coref_resolve_check':\n",
    "            del r['location']['location_type']\n",
    "            # r['location']['coref_resolve'] = d.get(\"location.coref_resolve.coref_resolve\")\n",
    "            # del d[\"location.coref_resolve.coref_resolve\"]\n",
    "        elif r[\"location\"][\"location_type\"] == \"REFERENCE_OBJECT\":\n",
    "            r[\"location\"][\"location_type\"] = \"REFERENCE_OBJECT\"\n",
    "            r[\"location\"][\"relative_direction\"] = d.get(\n",
    "                \"location.REFERENCE_OBJECT.relative_direction\"\n",
    "            )\n",
    "            # no key for EXACT\n",
    "            if r[\"location\"][\"relative_direction\"] in (\"EXACT\", \"Other\"):\n",
    "                del r[\"location\"][\"relative_direction\"]\n",
    "            d[\"location.REFERENCE_OBJECT.relative_direction\"] = None\n",
    "        r[\"location\"].update(process_dict(with_prefix(d, \"location.\")))\n",
    "        \n",
    "    for k, v in d.items():\n",
    "        #print(k, v)\n",
    "        if (\n",
    "            k == \"location\"\n",
    "            or k in ['COPY', 'coref_resolve_check', 'receiver', 'source']\n",
    "            or (k == \"relative_direction\" and v in (\"EXACT\", \"NEAR\", \"Other\"))\n",
    "        ):\n",
    "            continue\n",
    "        # handle span\n",
    "        if re.match(\"[^.]+.span#[0-9]+\", k):\n",
    "            prefix, rest = k.split(\".\", 1)\n",
    "            idx = int(rest.split(\"#\")[-1])\n",
    "            # print('here')\n",
    "            if prefix in r:\n",
    "                r[prefix].append([idx, idx])\n",
    "                r[prefix] = sorted(r[prefix], key=itemgetter(0))\n",
    "#                 a, b = r[prefix]\n",
    "#                 r[prefix] = [min(a, idx), max(b, idx)]  # expand span to include idx\n",
    "            else:\n",
    "                r[prefix] = [[idx, idx]]\n",
    "\n",
    "        # handle nested dict\n",
    "        elif \".\" in k:\n",
    "            prefix, rest = k.split(\".\", 1)\n",
    "            prefix_snake = snake_case(prefix)\n",
    "            #print(r)\n",
    "            r[prefix_snake] = r.get(prefix_snake, {})\n",
    "            r[prefix_snake].update(process_dict(with_prefix(d, prefix + \".\")))\n",
    "\n",
    "        # handle const value\n",
    "        else:\n",
    "            r[k] = v\n",
    "\n",
    "    return r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_put_memory(d):\n",
    "    return {}\n",
    "    \n",
    "\n",
    "def handle_commands(d):\n",
    "    output = {}\n",
    "    action_name = d[\"action_type\"]\n",
    "    formatted_dict = with_prefix(d, \"action_type.{}.\".format(action_name))\n",
    "    #print(formatted_dict)\n",
    "    child_d = process_dict(with_prefix(d, \"action_type.{}.\".format(action_name)))\n",
    "    #print(child_d)\n",
    "    # Fix Build/Freebuild mismatch\n",
    "    if child_d.get(\"FREEBUILD\") == \"FREEBUILD\":\n",
    "        action_name = 'FREEBUILD'\n",
    "    child_d.pop(\"FREEBUILD\", None)    \n",
    "    \n",
    "    if 'MOVE' in child_d:\n",
    "        if child_d['MOVE'] == 'yes':\n",
    "            action_name = 'MOVE'\n",
    "        elif child_d['MOVE'] == 'no':\n",
    "            action_name = 'DANCE'\n",
    "        child_d.pop('MOVE')\n",
    "    \n",
    "    \n",
    "    if formatted_dict.get('COPY', 'no') == 'yes':\n",
    "        action_name = 'COPY'\n",
    "        formatted_dict.pop('COPY')\n",
    "    \n",
    "    # add action type info\n",
    "    if 'TURN_CHECK' in child_d:\n",
    "        output['action_type'] = ['yes', child_d['TURN_CHECK'].lower()]\n",
    "        child_d.pop('TURN_CHECK')\n",
    "    else:\n",
    "        output['action_type'] = ['yes', action_name.lower()]\n",
    "    # add dialogue type info\n",
    "    if output['action_type'][1] == 'tag':\n",
    "        output['dialogue_type'] = ['yes', 'PUT_MEMORY']\n",
    "    else:\n",
    "        output['dialogue_type'] = ['yes', 'HUMAN_GIVE_COMMAND']\n",
    "    \n",
    "    if output['action_type'][1] == 'get' :\n",
    "        if 'receiver' in child_d:\n",
    "            if 'reference_object' in child_d['receiver']:\n",
    "                child_d['receiver_reference_object'] = child_d['receiver']['reference_object']\n",
    "            elif 'location' in child_d['receiver']:\n",
    "                child_d['receiver_location'] = child_d['receiver']['location']\n",
    "            child_d.pop('receiver')\n",
    "        if 'source' in child_d:\n",
    "            if 'reference_object' in child_d['source']:\n",
    "                child_d['source_reference_object'] = child_d['source']['reference_object']\n",
    "            elif 'location' in child_d['source']:\n",
    "                child_d['source_location'] = child_d['source']['location']\n",
    "            child_d.pop('source')\n",
    "    \n",
    "    for k, v in child_d.items():\n",
    "        if k in ['target_action_type', 'has_block_type', 'dance_type_name']:\n",
    "            output[k] = ['yes', v]\n",
    "            \n",
    "        elif type(v)==list or (k == 'receiver'):\n",
    "            output[k]= ['no', v]\n",
    "        else:\n",
    "            output[k] = ['yes', v]\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def process_result(full_d):\n",
    "    \n",
    "    worker_id = full_d[\"WorkerId\"]\n",
    "    d = with_prefix(full_d, \"Answer.root.\")\n",
    "    #print(d)\n",
    "    if not d:\n",
    "        return worker_id, {}, full_d['Input.command'].split()\n",
    "    try:\n",
    "        action = d[\"action_type\"]\n",
    "    except KeyError:\n",
    "        return worker_id, {}, full_d['Input.command'].split()\n",
    "\n",
    "    action_dict = handle_commands(d)\n",
    "\n",
    "    ##############\n",
    "    # repeat dict\n",
    "    ##############\n",
    "    #NOTE: this can probably loop over or hold indices of which specific action ?\n",
    "    if action_dict.get('dialogue_type', [None, None])[1] == 'HUMAN_GIVE_COMMAND':\n",
    "        if d.get(\"loop\") not in [None, \"Other\"]:\n",
    "            repeat_dict = process_repeat_dict(d)\n",
    "            if repeat_dict:\n",
    "                # Some turkers annotate a repeat dict for a repeat_count of 1.\n",
    "                # Don't include the repeat dict if that's the case\n",
    "                if repeat_dict.get('repeat_dir', None) == 'Other':\n",
    "                    repeat_dict.pop('repeat_dir')\n",
    "                if repeat_dict.get(\"repeat_count\"):\n",
    "                    a, b = repeat_dict[\"repeat_count\"][0]\n",
    "                    repeat_count_str = \" \".join(\n",
    "                        [full_d[\"Input.word{}\".format(x)] for x in range(a, b + 1)]\n",
    "                    )\n",
    "                    if repeat_count_str not in (\"a\", \"an\", \"one\", \"1\"):\n",
    "                        action_dict['repeat'] = ['yes', repeat_dict]\n",
    "    #                     action_val = list(action_dict.values())[0]  # check what this is\n",
    "    #                     if action_specific_dict.get(\"schematic\"):\n",
    "    #                         action_specific_dict[\"schematic\"][\"repeat\"] = repeat_dict\n",
    "    #                     elif action_specific_dict.get(\"reference_object\"):\n",
    "    #                         action_specific_dict[\"reference_object\"][\"repeat\"] = repeat_dict\n",
    "    #                     else:\n",
    "    #                         action_specific_dict[\"repeat\"] = repeat_dict\n",
    "                else:\n",
    "                    action_dict['repeat'] = ['yes', repeat_dict]\n",
    "\n",
    "            \n",
    "    ##################\n",
    "    # post-processing\n",
    "    ##################\n",
    "   \n",
    "\n",
    "    # Fix empty words messing up spans\n",
    "    words = [full_d[\"Input.word{}\".format(x)] for x in range(MAX_WORDS)]\n",
    "    action_dict, words = fix_spans_due_to_empty_words(action_dict, words)\n",
    "\n",
    "    return worker_id, action_dict, words\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_cnt_in_schematic(words, action_dict):\n",
    "    if 'repeat' not in action_dict:\n",
    "        return action_dict\n",
    "    repeat = action_dict['repeat']\n",
    "    val = []\n",
    "    if 'repeat_count' in repeat[1]:\n",
    "        val = repeat[1]['repeat_count']\n",
    "    elif 'repeat_key' in repeat[1] and repeat[1]['repeat_key'] == 'ALL':\n",
    "        if any(x in ['all', 'every', 'each'] for x in words):\n",
    "            if 'all' in words:\n",
    "                all_val = words.index('all')\n",
    "            elif 'each' in words:\n",
    "                all_val = words.index('each')\n",
    "            elif 'every' in words:\n",
    "                all_val = words.index('every')\n",
    "            val = [[all_val, all_val]]\n",
    "    else:\n",
    "        return action_dict\n",
    "    \n",
    "    for k, v in action_dict.items():\n",
    "        if k in ['schematic', 'reference_object']:\n",
    "            for i, meh in enumerate(v[1]):\n",
    "                # print(words, val)\n",
    "                if meh in val:\n",
    "                    v[1].pop(i)\n",
    "            action_dict[k] = [v[0], v[1]]\n",
    "    return action_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "get bricks\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n",
      "hand me that axe\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_reference_object': ['no', [[1, 1]]],\n",
      " 'reference_object': ['no', [[2, 2], [3, 3]]]}\n",
      "********************\n",
      "get wood\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n",
      "and add it to the tree\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_location': ['no', [[4, 4], [5, 5]]],\n",
      " 'reference_object': ['no', [[2, 2]]]}\n",
      "********************\n",
      "place block\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n",
      "push\n",
      "{'action_type': ['yes', 'otheraction'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "install dog\n",
      "{'action_type': ['yes', 'build'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'schematic': ['no', [[1, 1]]]}\n",
      "********************\n",
      "go get some stone and build a castle with it\n",
      "{'action_type': ['yes', 'composite_action'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "find a villiage\n",
      "{'action_type': ['yes', 'fill'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[2, 2]]]}\n",
      "********************\n",
      "bot bring me my red pants suit\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_reference_object': ['no', [[2, 2]]],\n",
      " 'reference_object': ['no', [[3, 3], [4, 4], [5, 5], [6, 6]]]}\n",
      "********************\n",
      "go deep into the cave and search for minerals\n",
      "{'action_type': ['yes', 'composite_action'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "give me stone\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_reference_object': ['no', [[1, 1]]],\n",
      " 'reference_object': ['no', [[2, 2]]]}\n",
      "********************\n",
      "find diamons\n",
      "{'action_type': ['yes', 'scout'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n",
      "move the book to inside the house\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_location': ['no', [[4, 4], [6, 6]]],\n",
      " 'reference_object': ['no', [[2, 2]]]}\n",
      "********************\n",
      "plant some seeds\n",
      "{'action_type': ['yes', 'build'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'repeat': ['yes', {'repeat_count': [[1, 1]], 'repeat_key': 'FOR'}],\n",
      " 'schematic': ['no', [[2, 2]]]}\n",
      "********************\n",
      "place five pigs\n",
      "{'action_type': ['yes', 'spawn'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[2, 2]]],\n",
      " 'repeat': ['yes', {'repeat_count': [[1, 1]], 'repeat_key': 'FOR'}]}\n",
      "********************\n",
      "give me a brick\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_reference_object': ['no', [[1, 1]]],\n",
      " 'reference_object': ['no', [[3, 3]]]}\n",
      "********************\n",
      "get plant\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n",
      "pick flower\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n",
      "give me wool\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_reference_object': ['no', [[1, 1]]],\n",
      " 'reference_object': ['no', [[2, 2]]]}\n",
      "********************\n",
      "give me a axe\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_reference_object': ['no', [[1, 1]]],\n",
      " 'reference_object': ['no', [[3, 3]]]}\n",
      "********************\n",
      "get wool\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n",
      "stay here\n",
      "{'action_type': ['yes', 'move'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'location': ['no', [[1, 1]]]}\n",
      "********************\n",
      "undo this wall i built and give me the resources\n",
      "{'action_type': ['yes', 'composite_action'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "make one tower green and the other tower turquoise\n",
      "{'action_type': ['yes', 'composite_action'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "go get flower\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[2, 2]]]}\n",
      "********************\n",
      "show fire\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n",
      "turn\n",
      "{'action_type': ['yes', 'look'], 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "do something\n",
      "{'action_type': ['yes', 'otheraction'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "grow\n",
      "{'action_type': ['yes', 'otheraction'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "place sandstone there\n",
      "{'action_type': ['yes', 'build'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'location': ['no', [[2, 2]]],\n",
      " 'schematic': ['no', [[1, 1]]]}\n",
      "********************\n",
      "stop in front of it\n",
      "{'action_type': ['yes', 'stop'], 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "find me a cow\n",
      "{'action_type': ['yes', 'scout'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[3, 3]]]}\n",
      "********************\n",
      "get that dustbin and take it to the road side for the collection guys\n",
      "{'action_type': ['yes', 'composite_action'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "hey bot can you grab some wood for me\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_reference_object': ['no', [[8, 8]]],\n",
      " 'reference_object': ['no', [[6, 6]]],\n",
      " 'repeat': ['yes', {'repeat_count': [[5, 5]], 'repeat_key': 'FOR'}]}\n",
      "********************\n",
      "get redstone from your inventory\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_location': ['no', [[4, 4]]],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n",
      "move towards right and lay down\n",
      "{'action_type': ['yes', 'composite_action'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "please make me a poach egg with a toast with avocado on top\n",
      "{'action_type': ['yes', 'build'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'schematic': ['no',\n",
      "               [[4, 4],\n",
      "                [5, 5],\n",
      "                [6, 6],\n",
      "                [8, 8],\n",
      "                [9, 9],\n",
      "                [10, 10],\n",
      "                [11, 11],\n",
      "                [12, 12]]]}\n",
      "********************\n",
      "go to the window and observe player in the distance\n",
      "{'action_type': ['yes', 'composite_action'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "dig all the herbs nearby and collect them to bring back to me so i can make potions to use in the game\n",
      "{'action_type': ['yes', 'composite_action'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "take some dirt from the pile\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[2, 2]]],\n",
      " 'source_location': ['no', [[5, 5]]]}\n",
      "********************\n",
      "look ut the window\n",
      "{'action_type': ['yes', 'look'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'facing': ['no', [[1, 1], [2, 2], [3, 3]]]}\n",
      "********************\n",
      "pick this up\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n",
      "chop a tree\n",
      "{'action_type': ['yes', 'destroy'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[2, 2]]]}\n",
      "********************\n",
      "walk over to the brown boulder and pick it up\n",
      "{'action_type': ['yes', 'composite_action'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND']}\n",
      "********************\n",
      "duck under the overhang of that wall to your right\n",
      "{'action_type': ['yes', 'dance'],\n",
      " 'dance_type_name': ['yes', [[0, 0]]],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'location': ['no', [[1, 1], [3, 3], [5, 5], [6, 6], [8, 8], [9, 9]]]}\n",
      "********************\n",
      "collect the mushrooms here\n",
      "{'action_type': ['yes', 'get'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'receiver_location': ['no', [[3, 3]]],\n",
      " 'reference_object': ['no', [[2, 2]]]}\n",
      "********************\n",
      "chase cow\n",
      "{'action_type': ['yes', 'move'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'location': ['no', [[1, 1]]],\n",
      " 'repeat': ['yes', {'stop_condition': {'condition_type': 'NEVER'}}]}\n",
      "********************\n",
      "place it\n",
      "{'action_type': ['yes', 'copy'],\n",
      " 'dialogue_type': ['yes', 'HUMAN_GIVE_COMMAND'],\n",
      " 'reference_object': ['no', [[1, 1]]]}\n",
      "********************\n"
     ]
    }
   ],
   "source": [
    "# convert csv to txt first\n",
    "from pprint import pprint\n",
    "\n",
    "#def process_results_with_agreements(f_name, num_agreements=1, debug=False, tsv=False, only_show_disagreements=False):\n",
    "num_agreements=2\n",
    "from pprint import pprint\n",
    "result_dict = {}\n",
    "folder_name = '/Users/kavyasrinet/Desktop/other_actions/5/toolA/'\n",
    "f_name = folder_name + 'out.csv'\n",
    "# folder_name = '/Users/kavyasrinet/Downloads/'\n",
    "# f_name = folder_name + 'out1.csv'\n",
    "only_show_disagreements=True\n",
    "with open(f_name, 'r') as f:\n",
    "    r = csv.DictReader(f)\n",
    "    for i, d in enumerate(r):\n",
    "        worker_id = d[\"WorkerId\"]\n",
    "        sentence = preprocess_chat(d['Input.command'])[0]\n",
    "        #if i == 1:\n",
    "        #print(d)\n",
    "        #if sentence == \"collect three iron ore blocks from the cave in front of you\":\n",
    "#       print(sentence)\n",
    "        _, action_dict, words = process_result(d)\n",
    "        a_dict = fix_cnt_in_schematic(words, action_dict)\n",
    "        print(sentence)\n",
    "        pprint(a_dict)\n",
    "        print(\"*\"*20)\n",
    "        if a_dict is None:\n",
    "            continue\n",
    "        command = \" \".join(words)\n",
    "        result = json.dumps(a_dict)\n",
    "        if command in result_dict:\n",
    "            if len(result_dict[command]) == 3:\n",
    "                print(command)\n",
    "                continue\n",
    "            result_dict[command].append(result)\n",
    "        else:\n",
    "            result_dict[command] = [result]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49\n"
     ]
    }
   ],
   "source": [
    "print(len(result_dict.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write to txt\n",
    "import json\n",
    "f_name = folder_name + 'out.txt'\n",
    "with open(f_name, 'w') as outfile:\n",
    "    for k, v in result_dict.items():\n",
    "        cmd = k\n",
    "        #print(len(v))\n",
    "        if len(v) == 1:\n",
    "            items = v[0] + \"\\t\" + v[0] + \"\\t\" + v[0]\n",
    "        else:\n",
    "            items =  \"\\t\".join(v) \n",
    "        outfile.write(cmd + \"\\t\" + items+\"\\n\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_definite_articles(cmd, d):\n",
    "    words = cmd.split()\n",
    "    if type(d) == str:\n",
    "        d = ast.literal_eval(d)\n",
    "    new_d = {}\n",
    "    # print(d)\n",
    "    for k, v in d.items():\n",
    "        # for level 1 \n",
    "        if type(v) == list and v[0] in ['yes', 'no']:\n",
    "            if type(v[1]) == list:\n",
    "                new_v = []\n",
    "                for span in v[1]:\n",
    "                    # span[0] and span[1] are the same\n",
    "                    if words[span[0]] in ['the', 'a', 'an']:\n",
    "                        continue\n",
    "                    new_v.append(span)\n",
    "                new_d[k] = [v[0], new_v]  \n",
    "            elif type(v[1]) == dict:\n",
    "                v_new = remove_definite_articles(cmd, v[1])\n",
    "                new_d[k] = [v[0], v_new]\n",
    "\n",
    "            else:\n",
    "                new_d[k] = v\n",
    "        # for recursion on normal internal dict\n",
    "        else:\n",
    "            if type(v) == list:\n",
    "                new_v = []\n",
    "                for span in v:\n",
    "                    # span[0] and span[1] are the same\n",
    "                    if words[span[0]] in ['the', 'a', 'an']:\n",
    "                        continue\n",
    "                    new_v.append(span)\n",
    "                new_d[k] = new_v  \n",
    "            elif type(v) == dict:\n",
    "                v_new = remove_definite_articles(cmd, v)\n",
    "                new_d[k] = v_new\n",
    "\n",
    "            else:\n",
    "                new_d[k] = v\n",
    "\n",
    "    return new_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49\n"
     ]
    }
   ],
   "source": [
    "# construct counter from txt\n",
    "result_counts = defaultdict(Counter)\n",
    "f_name = folder_name + 'out.txt'\n",
    "import ast\n",
    "with open(f_name) as in_data:\n",
    "    for line in in_data.readlines():\n",
    "        line = line.strip()\n",
    "        #print(len(line.split(\"\\t\")))\n",
    "        parts = line.split(\"\\t\")\n",
    "        if len(parts) == 4:\n",
    "            cmd, r1, r2, r3 = parts\n",
    "        elif len(parts) == 3: # for just one answer\n",
    "            cmd, r1, r2 = parts\n",
    "            r3 = r2 \n",
    "        else:\n",
    "            cmd, r = parts\n",
    "            r1, r2, r3 = r, r, r\n",
    "        for r in [r1, r2, r3]:\n",
    "            r_new = remove_definite_articles(cmd, r)\n",
    "            result_counts[cmd][json.dumps(r_new)] += 1\n",
    "print(len(result_counts.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "no_agreement = 0\n",
    "agreement = 0\n",
    "disagreement = defaultdict(Counter)\n",
    "only_show_disagreements = False\n",
    "all_agreements_dict = {}\n",
    "disagreements_dict = {}\n",
    "\n",
    "for command, counts in sorted(result_counts.items()):\n",
    "    if not any(v >= num_agreements for v in counts.values()):\n",
    "        if only_show_disagreements:\n",
    "            print(command)\n",
    "        disagreement[command] = counts\n",
    "        no_agreement += 1\n",
    "        for result, count in counts.items():\n",
    "            if command not in disagreements_dict:\n",
    "                disagreements_dict[command] = [result]\n",
    "            else:\n",
    "                disagreements_dict[command].append(result)\n",
    "        \n",
    "        continue\n",
    "    elif only_show_disagreements:\n",
    "        continue\n",
    "\n",
    "    for result, count in counts.items():\n",
    "        if count >= num_agreements:\n",
    "            all_agreements_dict[command] = result\n",
    "            agreement += 1\n",
    "            \n",
    "print(agreement)\n",
    "print(no_agreement)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write out agreements to a file\n",
    "## format is : command child dict\n",
    "ag = str(agreement)\n",
    "f = folder_name + ag + '_agreements.txt'\n",
    "with open(f, 'w') as outfile:\n",
    "    for k, v in all_agreements_dict.items():\n",
    "        cmd = k\n",
    "        outfile.write(cmd + \"\\t\" + v + \"\\n\") \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write disagreements to a file\n",
    "disag = str(no_agreement)\n",
    "f = folder_name + disag + '_disagreements.txt'\n",
    "with open(f, 'w') as outfile:\n",
    "    for k, v in disagreements_dict.items():\n",
    "        cmd = k\n",
    "        outfile.write(cmd + \"\\n\")\n",
    "        for item in v:\n",
    "            outfile.write(item + \"\\n\") \n",
    "        outfile.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# ONLY FOR DEBUGGING\n",
    "import ast\n",
    "from operator import itemgetter\n",
    "\n",
    "def resolve_spans(words, dicts):\n",
    "    result = {}\n",
    "    mapping_old_dicts = {}\n",
    "    for d, val in dicts.items():\n",
    "        new_d = {}\n",
    "        d = ast.literal_eval(d)\n",
    "        for k, v in d.items():\n",
    "            if type(v[1]) == list:\n",
    "                new_v = []\n",
    "                for item in v[1]:\n",
    "                    new_v.append(words[item[0]])\n",
    "                new_d[k] = [v[0], new_v]\n",
    "            elif k =='repeat':\n",
    "                #v[1] = ast.literal_eval(v[1])\n",
    "                if 'stop_condition' in v[1]:\n",
    "                    new_v = {}\n",
    "                    new_v['stop_condition'] = {}\n",
    "                    x = {}\n",
    "                    \n",
    "                    if 'condition_type' in v[1]['stop_condition']:\n",
    "                        x['condition_type'] = v[1]['stop_condition']['condition_type']\n",
    "                    new_vals = []\n",
    "                    if \"block_type\" in v[1][\"stop_condition\"]:\n",
    "                        for item in v[1]['stop_condition']['block_type']:\n",
    "                            new_vals.append(words[item[0]])\n",
    "                        x['block_type'] = new_vals\n",
    "                    elif 'condition_span' in v[1]['stop_condition']:\n",
    "                        for item in v[1]['stop_condition']['condition_span']:\n",
    "                            new_vals.append(words[item[0]])\n",
    "                        x['condition_span'] = new_vals\n",
    "                    new_v['stop_condition'] = x\n",
    "                    new_d['repeat'] = [v[0], new_v]\n",
    "                else:\n",
    "                    new_d[k] = v\n",
    "            else:\n",
    "                new_d[k] = v\n",
    "        result[str(new_d)] = val\n",
    "        mapping_old_dicts[str(new_d)] = str(d)\n",
    "    return result, mapping_old_dicts\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "for command, counts in disagreement.items():\n",
    "    words = command.split()\n",
    "    c, mapping_old_dicts = resolve_spans(words, counts)\n",
    "    print(command)\n",
    "    #print(counts)\n",
    "    # print(\"*\"*30)\n",
    "    for k, v in c.items():\n",
    "#         print(mapping_old_dicts[k])\n",
    "#         print(\"///\"*10)\n",
    "        pprint(ast.literal_eval(k))\n",
    "        print(\"-\"*10)\n",
    "    #print(c)\n",
    "    print(\"*\"*30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "#folder_name = '/Users/kavyasrinet/Github/minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/tool1/'\n",
    "\n",
    "with open(folder_name + 'all_agreements.txt', 'w') as f_out, \\\n",
    "     open(folder_name + ag +'_agreements.txt') as f1, \\\n",
    "    open(folder_name + disag + '_disagreements.txt') as f_in:\n",
    "    for line in f1.readlines():\n",
    "        cmd, out = line.strip().split(\"\\t\")\n",
    "        cmd  = preprocess_chat(cmd)[0]\n",
    "        f_out.write(cmd+\"\\t\"+ out+\"\\n\")\n",
    "    for line in f_in.readlines():\n",
    "#         print(len(line.strip().split(\"\\t\")))\n",
    "#         print(line.strip().split(\"\\t\"))\n",
    "        cmd, out = line.strip().split(\"\\t\")\n",
    "        f_out.write(cmd+\"\\t\"+ out+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
